const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

let sites_map = {'NFjQZzm': '简书'};

async function main() {
  const fileName = `jianshu.json`;
  let totalBlogs = [];

  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});

  page.on('pageerror', msg => {
    console.log(msg);
  });


  try { // 必须先登录
    await page.goto('https://www.tuicool.com/login', {waitUntil: 'domcontentloaded'});
    await page.waitFor(2000);
    await page.type('#xlEmail', 'yinhaixiang@qq.com', {delay: 11});
    await page.type('#xlPassword', 'tuicool', {delay: 1});
    await page.click('body > div.container-fluid > div.center_container.container-top > form > fieldset > div > div:nth-child(4) > button');
    await page.waitFor(2000);
  } catch (e) {
    console.log(e.message);
  }

  let blogs = null;

  for (let sites_id in sites_map) {
    for (let i = 1; i <= 20; i++) {
      let url = `https://www.tuicool.com/sites/NFjQZzm?pn=${i}`;
      await page.goto(url, {waitUntil: 'domcontentloaded'});
      await page.waitFor(2000);
      blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span9.contant > div:nth-child(2) > div.list_article > ol > li', async (eles, sub_tab) => {
        let tmpBlogs = [];
        eles.forEach(async ele => {
          let title = ele.querySelector('div > div > div.span9 > h4 > a').innerText;
          let content = ele.querySelector('div > div > div.span9 > h4 > a').href;
          tmpBlogs.push({
            title: title,
            content: content,
            tab: 'tuicool',
            sub_tab: sub_tab,
            source: sub_tab
          });
        });
        return tmpBlogs;
      }, sites_map[sites_id]);

      console.log(blogs);
      totalBlogs = totalBlogs.concat(blogs);
      console.log(sites_id + ',' + i + ',--------------');
    }
  }


  let writerStream = fs.createWriteStream(`/Users/yinhaixiang/WebstormProjects/egg-taosha/crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

  console.log(errLogs);

  // browser.close();
};


main();


